package cn.sheep.dmp.tools

import cn.sheep.dmp.utils.Tools
import org.apache.commons.lang.StringUtils
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Sheep.Old @ 64341393
  * Created 2018/3/29
  */
object Dict2Redis {


    def main(args: Array[String]): Unit = {

        val sparkConf = new SparkConf().setAppName("地域数据分布统计")
          .setMaster("local[*]")
          .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")

        val sc = new SparkContext(sparkConf)

        sc.textFile(Tools.load.getString("app.dict.path"))
          .foreachPartition(iter => {
              val jedis = JPools.getJedis
              jedis.select(4)
              iter.foreach{ line =>
                  val splited = line.split("\t", -1)

                  if (splited.size >=5) {
                      val appId = splited(4)
                      val appName = splited(1)

                      if(StringUtils.isNotEmpty(appId)
                        && StringUtils.isNotEmpty(appName)) jedis.hset("appdict", appId, appName)
                  }

              }

              jedis.close()
          })

        sc.stop()

    }

}
